{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from __future__ import (print_function, absolute_import)\n",
    "import os\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import keras\n",
    "import keras.backend as K\n",
    "from keras import datasets\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.models import load_model\n",
    "\n",
    "from models import CNN, VGG8\n",
    "from wide_resnet import WideResidualNetwork\n",
    "from pulearn.utils import synthesize_pn_labels, synthesize_pu_labels\n",
    "\n",
    "from keras.callbacks import (\n",
    "    ReduceLROnPlateau,\n",
    "    LearningRateScheduler,\n",
    "    CSVLogger,\n",
    "    EarlyStopping,\n",
    "    ModelCheckpoint)\n",
    "\n",
    "from keras_tqdm import TQDMNotebookCallback\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"]=\"0\"\n",
    "\n",
    "global _SESSION\n",
    "config = tf.ConfigProto(allow_soft_placement=True)\n",
    "config.gpu_options.allow_growth = True\n",
    "_SESSION = tf.Session(config=config)\n",
    "K.set_session(_SESSION)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CIFAR110"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_classes = 11\n",
    "batch_size = 128\n",
    "epochs = 200\n",
    "data_augmentation = True\n",
    "checkpoint = None\n",
    "# checkpoint = 'model_checkpoint_cifar10_wide_resnet.h5'\n",
    "title = 'cifar110_vgg8'\n",
    "# title = 'cifar110_wide_resnet'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "(x_train_10, y_train_10), (x_test_10, y_test_10) = datasets.cifar10.load_data()\n",
    "(x_train_100, y_train_100), (x_test_100, y_test_100) = datasets.cifar100.load_data()\n",
    "\n",
    "y_train_10 = y_train_10 + 1\n",
    "y_test_10 = y_test_10 + 1\n",
    "y_train_100[...] = 0\n",
    "y_test_100[...] = 0\n",
    "\n",
    "x_train = np.concatenate((x_train_10, x_train_100), axis=0).astype('float32')\n",
    "y_train = np.concatenate((y_train_10, y_train_100), axis=0).astype('int8')\n",
    "x_test = np.concatenate((x_test_10, x_test_100), axis=0).astype('float32')\n",
    "y_test = np.concatenate((y_test_10, y_test_100), axis=0).astype('int8')\n",
    "\n",
    "x_train = x_train.astype(K.floatx())\n",
    "x_test = x_test.astype(K.floatx())\n",
    "\n",
    "y_train = keras.utils.to_categorical(y_train, num_classes)\n",
    "y_test = keras.utils.to_categorical(y_test, num_classes)\n",
    "\n",
    "def normalize(x):\n",
    "    \"\"\"Substract mean and Divide by std.\"\"\"\n",
    "    x -= np.array([125.3, 123.0, 113.9], dtype=K.floatx())\n",
    "    x /= np.array([63.0, 62.1, 66.7], dtype=K.floatx())\n",
    "    return x\n",
    "\n",
    "\n",
    "x_train = normalize(x_train)\n",
    "x_test = normalize(x_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(100000, 32, 32, 3) (100000, 11)\n",
      "(20000, 32, 32, 3) (20000, 11)\n"
     ]
    }
   ],
   "source": [
    "print(x_train.shape, y_train.shape)\n",
    "print(x_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Positive (pct_missing=0.0):', 50000, ' -> ', 50000)\n",
      "('Positive (pct_missing=0.1):', 50000, ' -> ', 44990)\n",
      "('Positive (pct_missing=0.2):', 50000, ' -> ', 39992)\n",
      "('Positive (pct_missing=0.3):', 50000, ' -> ', 34902)\n",
      "('Positive (pct_missing=0.4):', 50000, ' -> ', 30043)\n",
      "('Positive (pct_missing=0.5):', 50000, ' -> ', 24968)\n",
      "('Positive (pct_missing=0.6):', 50000, ' -> ', 19937)\n",
      "('Positive (pct_missing=0.7):', 50000, ' -> ', 14896)\n",
      "('Positive (pct_missing=0.8):', 50000, ' -> ', 10153)\n",
      "('Positive (pct_missing=0.9):', 50000, ' -> ', 4902)\n",
      "('Positive (pct_missing=1.0):', 50000, ' -> ', 0)\n",
      "('Positive vs. Negative (pct=0.0):', 50000, ' vs.', 50000)\n",
      "('Positive vs. Negative (pct=0.1):', 44990, ' vs.', 45014)\n",
      "('Positive vs. Negative (pct=0.2):', 39992, ' vs.', 40104)\n",
      "('Positive vs. Negative (pct=0.3):', 34902, ' vs.', 34988)\n",
      "('Positive vs. Negative (pct=0.4):', 30043, ' vs.', 30178)\n",
      "('Positive vs. Negative (pct=0.5):', 24968, ' vs.', 24841)\n",
      "('Positive vs. Negative (pct=0.6):', 19937, ' vs.', 20106)\n",
      "('Positive vs. Negative (pct=0.7):', 14896, ' vs.', 15058)\n",
      "('Positive vs. Negative (pct=0.8):', 10153, ' vs.', 10164)\n",
      "('Positive vs. Negative (pct=0.9):', 4902, ' vs.', 5033)\n",
      "('Positive vs. Negative (pct=1.0):', 0, ' vs.', 0)\n",
      "(80096, 32, 32, 3) (80096, 11)\n",
      "class_weight {0: 0.99860363056054258, 1: 1.0014002800560111, 2: 1.0014002800560111, 3: 1.0014002800560111, 4: 1.0014002800560111, 5: 1.0014002800560111, 6: 1.0014002800560111, 7: 1.0014002800560111, 8: 1.0014002800560111, 9: 1.0014002800560111, 10: 1.0014002800560111}\n"
     ]
    }
   ],
   "source": [
    "pct_missing = 0.2\n",
    "y_train_pu = synthesize_pu_labels(y_train, random_state=42, verbose=True)\n",
    "y_train_pn = synthesize_pn_labels(y_train, random_state=42, verbose=True)\n",
    "x_train_enc = x_train[y_train_pn[pct_missing],...]\n",
    "y_train_enc = y_train[y_train_pn[pct_missing],...]\n",
    "print(x_train_enc.shape, y_train_enc.shape)\n",
    "\n",
    "from sklearn.utils import class_weight\n",
    "\n",
    "y_train_dec = 1 - y_train_enc[:, 0]\n",
    "class_weights = class_weight.compute_class_weight('balanced',\n",
    "                                                 [0, 1],\n",
    "                                                 y_train_dec)\n",
    "# class_weights = [1., 2.]\n",
    "class_weight = {}\n",
    "class_weight[0] = class_weights[0]\n",
    "for k in range(1, num_classes):\n",
    "    class_weight[k] = class_weights[1]\n",
    "\n",
    "\n",
    "print('class_weight', class_weight)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "conv2d_19 (Conv2D)           (None, 32, 32, 32)        896       \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_19 (LeakyReLU)   (None, 32, 32, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_20 (Conv2D)           (None, 32, 32, 32)        9248      \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_20 (LeakyReLU)   (None, 32, 32, 32)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_10 (MaxPooling (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "dropout_13 (Dropout)         (None, 16, 16, 32)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_21 (Conv2D)           (None, 16, 16, 64)        18496     \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_21 (LeakyReLU)   (None, 16, 16, 64)        0         \n",
      "_________________________________________________________________\n",
      "conv2d_22 (Conv2D)           (None, 16, 16, 64)        36928     \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_22 (LeakyReLU)   (None, 16, 16, 64)        0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_11 (MaxPooling (None, 8, 8, 64)          0         \n",
      "_________________________________________________________________\n",
      "dropout_14 (Dropout)         (None, 8, 8, 64)          0         \n",
      "_________________________________________________________________\n",
      "conv2d_23 (Conv2D)           (None, 8, 8, 128)         73856     \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_23 (LeakyReLU)   (None, 8, 8, 128)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_24 (Conv2D)           (None, 8, 8, 128)         147584    \n",
      "_________________________________________________________________\n",
      "leaky_re_lu_24 (LeakyReLU)   (None, 8, 8, 128)         0         \n",
      "_________________________________________________________________\n",
      "max_pooling2d_12 (MaxPooling (None, 4, 4, 128)         0         \n",
      "_________________________________________________________________\n",
      "dropout_15 (Dropout)         (None, 4, 4, 128)         0         \n",
      "_________________________________________________________________\n",
      "flatten_4 (Flatten)          (None, 2048)              0         \n",
      "_________________________________________________________________\n",
      "dense_7 (Dense)              (None, 512)               1049088   \n",
      "_________________________________________________________________\n",
      "activation_7 (Activation)    (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dropout_16 (Dropout)         (None, 512)               0         \n",
      "_________________________________________________________________\n",
      "dense_8 (Dense)              (None, 11)                5643      \n",
      "_________________________________________________________________\n",
      "activation_8 (Activation)    (None, 11)                0         \n",
      "=================================================================\n",
      "Total params: 1,341,739\n",
      "Trainable params: 1,341,739\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "if title == 'cifar110_vgg8':\n",
    "    model = VGG8(input_shape=x_train_enc.shape[1:], num_classes=num_classes)\n",
    "elif title == 'cifar110_wide_resnet':\n",
    "    model = WideResidualNetwork(depth=28, width=8, dropout_rate=0.5,\n",
    "                                classes=num_classes, include_top=True,\n",
    "                                weights=None)\n",
    "if checkpoint is not None:\n",
    "    model = load_model(checkpoint)\n",
    "    \n",
    "\n",
    "model.compile(loss='categorical_crossentropy',\n",
    "              optimizer='adam',\n",
    "              metrics=['accuracy'])\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Checkpoint\n",
    "checkpointer = ModelCheckpoint(\n",
    "    filepath=\"model_checkpoint_{}_pn_{}.h5\".format(title, pct_missing),\n",
    "    verbose=1,\n",
    "    save_best_only=True)\n",
    "\n",
    "# csvlogger\n",
    "csv_logger = CSVLogger(\n",
    "    'csv_logger_{}_pn_{}.csv'.format(title, pct_missing))\n",
    "\n",
    "# EarlyStopping\n",
    "early_stopper = EarlyStopping(monitor='val_loss',\n",
    "                              min_delta=0.001,\n",
    "                              patience=50)\n",
    "# Reduce lr with schedule\n",
    "# def schedule(epoch):\n",
    "#     lr = K.get_value(model.optimizer.lr)\n",
    "#     if epoch in [60, 120, 160]:\n",
    "#         lr = lr * np.sqrt(0.1)\n",
    "#     return lr\n",
    "# lr_scheduler = LearningRateScheduler(schedule)\n",
    "\n",
    "# Reduce lr on plateau\n",
    "lr_reducer = ReduceLROnPlateau(factor=np.sqrt(0.1),\n",
    "                               cooldown=0,\n",
    "                               patience=20,\n",
    "                               min_lr=0.3e-6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training 80096 samples\n",
      "Using real-time data augmentation.\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48494fbe2db54ebf8902433c7cff4dc2"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "48e31297d6914e7e9b2af567056f7cee"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 1.4373 - acc: 0.5454Epoch 00000: val_loss improved from inf to 1.09401, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 28s - loss: 1.4365 - acc: 0.5455 - val_loss: 1.0940 - val_acc: 0.6323\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ea0f3bce7eda4e96b2a8ad0a0c0d3d06"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 2/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 1.1244 - acc: 0.6227Epoch 00001: val_loss improved from 1.09401 to 0.92825, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 26s - loss: 1.1241 - acc: 0.6229 - val_loss: 0.9282 - val_acc: 0.6847\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e1a80ec434bc424699abecfabc57c5d7"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 3/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 1.0133 - acc: 0.6557Epoch 00002: val_loss improved from 0.92825 to 0.84536, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 28s - loss: 1.0135 - acc: 0.6556 - val_loss: 0.8454 - val_acc: 0.7066\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b1dc8385d1414bc59f1a802faf3f762e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 4/200\n",
      "778/781 [============================>.] - ETA: 0s - loss: 0.9618 - acc: 0.6716Epoch 00003: val_loss improved from 0.84536 to 0.82294, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 26s - loss: 0.9614 - acc: 0.6718 - val_loss: 0.8229 - val_acc: 0.7165\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9db5a58aa87d4f8b91a96ef91a7058d3"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 5/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.9131 - acc: 0.6887Epoch 00004: val_loss improved from 0.82294 to 0.77319, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 25s - loss: 0.9131 - acc: 0.6887 - val_loss: 0.7732 - val_acc: 0.7368\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6a20cb1cad83472c8b4fcc3d0c1a95d6"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 6/200\n",
      "778/781 [============================>.] - ETA: 0s - loss: 0.8871 - acc: 0.6977Epoch 00005: val_loss improved from 0.77319 to 0.74486, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 24s - loss: 0.8872 - acc: 0.6977 - val_loss: 0.7449 - val_acc: 0.7441\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "97f9e5c9d82646009ba20f9830c8de86"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 7/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.8656 - acc: 0.7053Epoch 00006: val_loss improved from 0.74486 to 0.72714, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 26s - loss: 0.8655 - acc: 0.7054 - val_loss: 0.7271 - val_acc: 0.7497\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "11aa0dc1309e419183d1a0d965fe03f0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 8/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.8439 - acc: 0.7113Epoch 00007: val_loss improved from 0.72714 to 0.70402, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 26s - loss: 0.8436 - acc: 0.7114 - val_loss: 0.7040 - val_acc: 0.7590\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1c9008aa0723425584bd00cf805f1f05"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 9/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.8320 - acc: 0.7167Epoch 00008: val_loss improved from 0.70402 to 0.69023, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 28s - loss: 0.8323 - acc: 0.7166 - val_loss: 0.6902 - val_acc: 0.7652\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7c0af973de14893bd579e00fa99ac49"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 10/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.8121 - acc: 0.7224Epoch 00009: val_loss did not improve\n",
      "781/781 [==============================] - 25s - loss: 0.8121 - acc: 0.7223 - val_loss: 0.7032 - val_acc: 0.7584\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "559101fd3f0c4afe9338788ecab91226"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 11/200\n",
      "778/781 [============================>.] - ETA: 0s - loss: 0.8115 - acc: 0.7228Epoch 00010: val_loss improved from 0.69023 to 0.66027, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 25s - loss: 0.8114 - acc: 0.7229 - val_loss: 0.6603 - val_acc: 0.7710\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fc09539eb92e4d76bf1a66d0d7970ab4"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 12/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.7955 - acc: 0.7282Epoch 00011: val_loss improved from 0.66027 to 0.65253, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 26s - loss: 0.7955 - acc: 0.7282 - val_loss: 0.6525 - val_acc: 0.7775\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1e2c793028344644aa372306437ad42f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.7867 - acc: 0.7307Epoch 00012: val_loss improved from 0.65253 to 0.64603, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 27s - loss: 0.7869 - acc: 0.7306 - val_loss: 0.6460 - val_acc: 0.7827\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "545c0c46736b4296af386423a0ed72e5"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 14/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7776 - acc: 0.7360Epoch 00013: val_loss improved from 0.64603 to 0.64146, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 25s - loss: 0.7775 - acc: 0.7360 - val_loss: 0.6415 - val_acc: 0.7831\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1da8d046f2ea47e98a6b09a62453a1e2"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 15/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.7718 - acc: 0.7368Epoch 00014: val_loss improved from 0.64146 to 0.64133, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 28s - loss: 0.7715 - acc: 0.7369 - val_loss: 0.6413 - val_acc: 0.7827\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "becaff0770c64b62a142c0b1e4ca92ac"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 16/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.7699 - acc: 0.7361Epoch 00015: val_loss improved from 0.64133 to 0.63903, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 28s - loss: 0.7698 - acc: 0.7362 - val_loss: 0.6390 - val_acc: 0.7820\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e5c0719e187d4d6aa3352beca446f540"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 17/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7598 - acc: 0.7416Epoch 00016: val_loss improved from 0.63903 to 0.62509, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 24s - loss: 0.7599 - acc: 0.7416 - val_loss: 0.6251 - val_acc: 0.7876\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "591fa1841fe7452bb980c8219f0b1772"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 18/200\n",
      "778/781 [============================>.] - ETA: 0s - loss: 0.7522 - acc: 0.7438Epoch 00017: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.7523 - acc: 0.7439 - val_loss: 0.6251 - val_acc: 0.7873\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "df029cf3b3cc46ae8e1fa14efaf20b7a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 19/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7510 - acc: 0.7437Epoch 00018: val_loss improved from 0.62509 to 0.62212, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 27s - loss: 0.7508 - acc: 0.7437 - val_loss: 0.6221 - val_acc: 0.7879\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a938df2c7764d2b8ecac5c62be2332e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 20/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.7525 - acc: 0.7437Epoch 00019: val_loss improved from 0.62212 to 0.61793, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 25s - loss: 0.7524 - acc: 0.7437 - val_loss: 0.6179 - val_acc: 0.7882\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef18557c245d40a9ab34c80a2f4d7ac1"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 21/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.7385 - acc: 0.7482Epoch 00020: val_loss improved from 0.61793 to 0.60875, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 25s - loss: 0.7385 - acc: 0.7482 - val_loss: 0.6087 - val_acc: 0.7920\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "309ce7d4f6c346dca86ec50a0068c1a3"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 22/200\n",
      "778/781 [============================>.] - ETA: 0s - loss: 0.7406 - acc: 0.7477Epoch 00021: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.7403 - acc: 0.7477 - val_loss: 0.6255 - val_acc: 0.7873\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "87c3a3092d1f4d4994aab80f3b7e6df9"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 23/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7353 - acc: 0.7494Epoch 00022: val_loss improved from 0.60875 to 0.59480, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 25s - loss: 0.7353 - acc: 0.7495 - val_loss: 0.5948 - val_acc: 0.7954\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ed9530835d85408d934dbbeb3de66607"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 24/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7318 - acc: 0.7504Epoch 00023: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.7318 - acc: 0.7504 - val_loss: 0.6258 - val_acc: 0.7893\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d4a73f669b364f7cac4c009b591aae7a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 25/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.7255 - acc: 0.7536Epoch 00024: val_loss improved from 0.59480 to 0.59463, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 26s - loss: 0.7254 - acc: 0.7537 - val_loss: 0.5946 - val_acc: 0.7952\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "886f2feb7eb34b62b829db834a032d6e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 26/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7294 - acc: 0.7516Epoch 00025: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.7295 - acc: 0.7516 - val_loss: 0.6084 - val_acc: 0.7918\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8b2543bf89664e09a6eed2c09bd14a68"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 27/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7243 - acc: 0.7533Epoch 00026: val_loss improved from 0.59463 to 0.58437, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 25s - loss: 0.7243 - acc: 0.7534 - val_loss: 0.5844 - val_acc: 0.8000\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4401655ee9c0480ba87d9529dde6c663"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 28/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7191 - acc: 0.7550Epoch 00027: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.7192 - acc: 0.7550 - val_loss: 0.5974 - val_acc: 0.7970\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "37ef69797a224a82a6b17b2d96402b5b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 29/200\n",
      "778/781 [============================>.] - ETA: 0s - loss: 0.7123 - acc: 0.7593Epoch 00028: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.7125 - acc: 0.7592 - val_loss: 0.6000 - val_acc: 0.7989\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "449e3871bc7d4cf484dbd9cbf58e6075"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 30/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7166 - acc: 0.7550Epoch 00029: val_loss did not improve\n",
      "781/781 [==============================] - 24s - loss: 0.7167 - acc: 0.7549 - val_loss: 0.6026 - val_acc: 0.7929\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4093601be8e3471e90e3bdc3519ab63f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 31/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.7169 - acc: 0.7559Epoch 00030: val_loss improved from 0.58437 to 0.58028, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 26s - loss: 0.7170 - acc: 0.7558 - val_loss: 0.5803 - val_acc: 0.8016\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d1247ea95a4f47b69043c5be120ca7c2"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 32/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7056 - acc: 0.7607Epoch 00031: val_loss improved from 0.58028 to 0.56718, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 27s - loss: 0.7059 - acc: 0.7606 - val_loss: 0.5672 - val_acc: 0.8076\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5e6f8e3a63a745ae808679f8e3fbb747"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 33/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7098 - acc: 0.7600Epoch 00032: val_loss did not improve\n",
      "781/781 [==============================] - 25s - loss: 0.7099 - acc: 0.7600 - val_loss: 0.6042 - val_acc: 0.7935\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a40a5f4ca3e342aebf859e4d84b752cb"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 34/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7072 - acc: 0.7600Epoch 00033: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.7070 - acc: 0.7600 - val_loss: 0.5970 - val_acc: 0.7971\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b5d7c1a4d0054a928608e39418417284"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 35/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7066 - acc: 0.7607Epoch 00034: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.7067 - acc: 0.7607 - val_loss: 0.6171 - val_acc: 0.7915\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "12001abb63ae41e582e3a23e756d4928"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 36/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7037 - acc: 0.7625Epoch 00035: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.7036 - acc: 0.7625 - val_loss: 0.5844 - val_acc: 0.8016\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "65c25ba5624b460bb441b04913e3ee20"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 37/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.7066 - acc: 0.7614Epoch 00036: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.7068 - acc: 0.7613 - val_loss: 0.5866 - val_acc: 0.8020\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f08675069fa441e396dc8a5c4aaadd25"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 38/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6983 - acc: 0.7633Epoch 00037: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.6984 - acc: 0.7633 - val_loss: 0.5820 - val_acc: 0.8032\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "41674d8b57bf4ea8b8bfa67fbf790e1b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 39/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6961 - acc: 0.7656Epoch 00038: val_loss improved from 0.56718 to 0.55949, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 28s - loss: 0.6961 - acc: 0.7656 - val_loss: 0.5595 - val_acc: 0.8076\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05f3084eea3e4caa871a40097424a13a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 40/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6947 - acc: 0.7655Epoch 00039: val_loss improved from 0.55949 to 0.55717, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 26s - loss: 0.6945 - acc: 0.7655 - val_loss: 0.5572 - val_acc: 0.8114\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b98cd9307ba14dcda45e5cf9dde87aad"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 41/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6919 - acc: 0.7659Epoch 00040: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.6918 - acc: 0.7660 - val_loss: 0.5619 - val_acc: 0.8058\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0510958299344e879e13ba7719cde663"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 42/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6951 - acc: 0.7642Epoch 00041: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.6950 - acc: 0.7643 - val_loss: 0.5635 - val_acc: 0.8083\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ae23588cc7f0439c90754dc76ab18ff3"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 43/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6964 - acc: 0.7646Epoch 00042: val_loss improved from 0.55717 to 0.55565, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 24s - loss: 0.6962 - acc: 0.7647 - val_loss: 0.5556 - val_acc: 0.8101\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a507a42af38446b8bbc6b0079c66904"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 44/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6932 - acc: 0.7652Epoch 00043: val_loss improved from 0.55565 to 0.54838, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 27s - loss: 0.6933 - acc: 0.7653 - val_loss: 0.5484 - val_acc: 0.8158\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b453a43cb7ab4bcd81bb4af761f92611"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 45/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6935 - acc: 0.7654Epoch 00044: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.6937 - acc: 0.7655 - val_loss: 0.5861 - val_acc: 0.8020\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "98adea91350f4ff9947537105f7e8f2a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 46/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6863 - acc: 0.7689Epoch 00045: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.6865 - acc: 0.7688 - val_loss: 0.5546 - val_acc: 0.8103\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6c2d5246f4364ea0b7664949221ab44c"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 47/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6902 - acc: 0.7659Epoch 00046: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.6902 - acc: 0.7659 - val_loss: 0.5563 - val_acc: 0.8099\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4076d864f4d14e04ba06949a169935ed"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 48/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6963 - acc: 0.7659Epoch 00047: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.6961 - acc: 0.7659 - val_loss: 0.5563 - val_acc: 0.8157\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9c8386025a4549fdb7315ba822dabc65"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 49/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6843 - acc: 0.7682Epoch 00048: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.6843 - acc: 0.7682 - val_loss: 0.5628 - val_acc: 0.8095\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bb51e48b9a2b4ec28bb895a9434915e9"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 50/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6904 - acc: 0.7678Epoch 00049: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.6902 - acc: 0.7679 - val_loss: 0.5774 - val_acc: 0.8037\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0469682bec34fb9b9246a6767db659d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 51/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6912 - acc: 0.7676- ETEpoch 00050: val_loss did not improve\n",
      "781/781 [==============================] - 25s - loss: 0.6911 - acc: 0.7677 - val_loss: 0.5611 - val_acc: 0.8112\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d77f291c1c094d4a9e13f6c9058a6717"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 52/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6801 - acc: 0.7705Epoch 00051: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.6804 - acc: 0.7704 - val_loss: 0.5498 - val_acc: 0.8124\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1af2dcb7873a434d882b361f60923d5b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 53/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6854 - acc: 0.7669Epoch 00052: val_loss improved from 0.54838 to 0.54720, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 28s - loss: 0.6857 - acc: 0.7668 - val_loss: 0.5472 - val_acc: 0.8158\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2c8d9a2a793a430198f63d8592bbafa3"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 54/200\n",
      "778/781 [============================>.] - ETA: 0s - loss: 0.6766 - acc: 0.7727Epoch 00053: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.6765 - acc: 0.7728 - val_loss: 0.5660 - val_acc: 0.8083\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "05dcec83d01d45399bd6a61d2f18c8bb"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 55/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6827 - acc: 0.7692Epoch 00054: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.6827 - acc: 0.7692 - val_loss: 0.5700 - val_acc: 0.8067\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7cb87da98db54e28a6d8accec44d6c4f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 56/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6823 - acc: 0.7708Epoch 00055: val_loss did not improve\n",
      "781/781 [==============================] - 28s - loss: 0.6825 - acc: 0.7707 - val_loss: 0.5580 - val_acc: 0.8072\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31ccb8465c1f482ba65cd2abb4a010ac"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 57/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6828 - acc: 0.7715Epoch 00056: val_loss did not improve\n",
      "781/781 [==============================] - 24s - loss: 0.6826 - acc: 0.7716 - val_loss: 0.5510 - val_acc: 0.8155\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ca725c67d70406fbcc92bf8e41461fa"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 58/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6788 - acc: 0.7718Epoch 00057: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.6787 - acc: 0.7718 - val_loss: 0.5522 - val_acc: 0.8143\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bfded6c403eb484e9dc2bf962165b594"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 59/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6818 - acc: 0.7705Epoch 00058: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.6819 - acc: 0.7704 - val_loss: 0.5503 - val_acc: 0.8157\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9ec57cc56a46479a860519e32a79efea"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 60/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6829 - acc: 0.7689Epoch 00059: val_loss improved from 0.54720 to 0.53184, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 27s - loss: 0.6826 - acc: 0.7690 - val_loss: 0.5318 - val_acc: 0.8220\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a2941cb0515e4847bc6f01687cc57db5"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 61/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6678 - acc: 0.7744Epoch 00060: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.6678 - acc: 0.7744 - val_loss: 0.5649 - val_acc: 0.8102\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7d31350ae22549ba9ccb09a898766622"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 62/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6798 - acc: 0.7711Epoch 00061: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.6797 - acc: 0.7711 - val_loss: 0.5626 - val_acc: 0.8112\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3ee03c568e554eb7a23a4a750552fd18"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 63/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6732 - acc: 0.7736Epoch 00062: val_loss did not improve\n",
      "781/781 [==============================] - 24s - loss: 0.6732 - acc: 0.7736 - val_loss: 0.5553 - val_acc: 0.8145\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa84f4ab09bb48e5ae8788ed7db9de67"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 64/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6779 - acc: 0.7722Epoch 00063: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.6788 - acc: 0.7721 - val_loss: 0.5653 - val_acc: 0.8113\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "897e82b9d95d44a8a651d4bb2bcf842b"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 65/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6746 - acc: 0.7734Epoch 00064: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.6745 - acc: 0.7735 - val_loss: 0.5572 - val_acc: 0.8155\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "558fd371401e4a6ba017fd3a9bb95b42"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 66/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6718 - acc: 0.7739- ETA: 1s - loss: 0.6732 - acc - ETA: 1s - loss: 0.672Epoch 00065: val_loss did not improve\n",
      "781/781 [==============================] - 24s - loss: 0.6720 - acc: 0.7738 - val_loss: 0.5330 - val_acc: 0.8177\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3dd2cffeef39484eaa37c5ec8e700ef5"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 67/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6731 - acc: 0.7737Epoch 00066: val_loss did not improve\n",
      "781/781 [==============================] - 25s - loss: 0.6729 - acc: 0.7738 - val_loss: 0.5635 - val_acc: 0.8096\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b293ad9d66a403c8e03eb687bbe1816"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 68/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6794 - acc: 0.7726Epoch 00067: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.6796 - acc: 0.7726 - val_loss: 0.5452 - val_acc: 0.8136\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "01208f3ae0f44412be15b2cd521c9ccd"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 69/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6713 - acc: 0.7746Epoch 00068: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.6720 - acc: 0.7744 - val_loss: 0.5629 - val_acc: 0.8096\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e8a9e76a172645649d44fa15663fb44d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 70/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6747 - acc: 0.7739Epoch 00069: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.6749 - acc: 0.7738 - val_loss: 0.5403 - val_acc: 0.8166\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9220e78f249541c9a96fccb89de2ecd5"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 71/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6692 - acc: 0.7746Epoch 00070: val_loss did not improve\n",
      "781/781 [==============================] - 24s - loss: 0.6692 - acc: 0.7747 - val_loss: 0.5461 - val_acc: 0.8186\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6d0b264884de4088b1a5cb72a5dbe4c3"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 72/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6741 - acc: 0.7747Epoch 00071: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.6740 - acc: 0.7747 - val_loss: 0.5467 - val_acc: 0.8143\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ee4e4608cddd496caebb8da71509f702"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 73/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6717 - acc: 0.7749Epoch 00072: val_loss did not improve\n",
      "781/781 [==============================] - 28s - loss: 0.6718 - acc: 0.7749 - val_loss: 0.5530 - val_acc: 0.8137\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2eba3a83504949e1a70599683281f1d9"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 74/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6662 - acc: 0.7767Epoch 00073: val_loss did not improve\n",
      "781/781 [==============================] - 28s - loss: 0.6661 - acc: 0.7768 - val_loss: 0.5519 - val_acc: 0.8158\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "82974fa8ba164e49841308747266cfaf"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 75/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6754 - acc: 0.7742Epoch 00074: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.6756 - acc: 0.7741 - val_loss: 0.5700 - val_acc: 0.8116\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a59ed5b500c14680937adacf5b9cc991"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 76/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6697 - acc: 0.7747Epoch 00075: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.6697 - acc: 0.7746 - val_loss: 0.5390 - val_acc: 0.8136\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6e37683b36c24627be4f84dcf626c4e1"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 77/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6717 - acc: 0.7740Epoch 00076: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.6716 - acc: 0.7740 - val_loss: 0.5649 - val_acc: 0.8165\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8e06a438b9e14db8b1d3e8cc62afe3c7"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 78/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6687 - acc: 0.7764Epoch 00077: val_loss did not improve\n",
      "781/781 [==============================] - 28s - loss: 0.6685 - acc: 0.7766 - val_loss: 0.5412 - val_acc: 0.8197\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a32b8b577a364d669c64c02f5aecdb60"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 79/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.6688 - acc: 0.7763Epoch 00078: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.6688 - acc: 0.7763 - val_loss: 0.5425 - val_acc: 0.8210\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e5d39e140e89408da510080fbeca82fb"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 80/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6666 - acc: 0.7757- ETA: 0s - loss: 0.6667 - acc: 0.7Epoch 00079: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.6665 - acc: 0.7757 - val_loss: 0.5605 - val_acc: 0.8114\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5dfec00596b7463bb64b1fd00160c2f7"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 81/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.6705 - acc: 0.7763Epoch 00080: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.6706 - acc: 0.7763 - val_loss: 0.5366 - val_acc: 0.8228\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de26598435b448f597114adcea9c7f38"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 82/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.5713 - acc: 0.8066- ETA: 2s - loss:  - ETA: 0s - loss: 0.571Epoch 00081: val_loss improved from 0.53184 to 0.47340, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 26s - loss: 0.5711 - acc: 0.8067 - val_loss: 0.4734 - val_acc: 0.8437\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3175ec5a4d5d40c2966471e0b4f1baa3"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 83/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.5505 - acc: 0.8111Epoch 00082: val_loss improved from 0.47340 to 0.46890, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 26s - loss: 0.5505 - acc: 0.8112 - val_loss: 0.4689 - val_acc: 0.8431\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "49360cddf48141688a623d717963e3c1"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 84/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.5364 - acc: 0.8181Epoch 00083: val_loss improved from 0.46890 to 0.46208, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 26s - loss: 0.5364 - acc: 0.8181 - val_loss: 0.4621 - val_acc: 0.8433\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "656fa2a61e5a490d8b3cca38d557c0a5"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 85/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.5317 - acc: 0.8177Epoch 00084: val_loss improved from 0.46208 to 0.45947, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 26s - loss: 0.5316 - acc: 0.8178 - val_loss: 0.4595 - val_acc: 0.8431\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f0f20fd8b1f64e158fc76d39748f1209"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 86/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.5272 - acc: 0.8185Epoch 00085: val_loss improved from 0.45947 to 0.45068, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 25s - loss: 0.5272 - acc: 0.8185 - val_loss: 0.4507 - val_acc: 0.8476\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "531adfb5c5b347238bfc545bb9284bda"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 87/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.5181 - acc: 0.8224- ETA: 1s - Epoch 00086: val_loss improved from 0.45068 to 0.45036, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 27s - loss: 0.5181 - acc: 0.8224 - val_loss: 0.4504 - val_acc: 0.8484\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eae75b3d12b74f7184a931ba09421aab"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 88/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.5162 - acc: 0.8225Epoch 00087: val_loss did not improve\n",
      "781/781 [==============================] - 28s - loss: 0.5163 - acc: 0.8225 - val_loss: 0.4607 - val_acc: 0.8474\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3402138073534d6584e39ec636746e06"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 89/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.5168 - acc: 0.8229Epoch 00088: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.5169 - acc: 0.8229 - val_loss: 0.4609 - val_acc: 0.8468\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "68037edfbbab454e89081780063c4df4"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 90/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.5126 - acc: 0.8229Epoch 00089: val_loss did not improve\n",
      "781/781 [==============================] - 25s - loss: 0.5127 - acc: 0.8229 - val_loss: 0.4528 - val_acc: 0.8498\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "be259dcd7c5d4c1fb7e5522031d47424"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 91/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.5087 - acc: 0.8254Epoch 00090: val_loss improved from 0.45036 to 0.44241, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 25s - loss: 0.5088 - acc: 0.8253 - val_loss: 0.4424 - val_acc: 0.8513\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "eb124f0cc6644dd79cd39d82702724ea"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 92/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.5051 - acc: 0.8273Epoch 00091: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.5051 - acc: 0.8273 - val_loss: 0.4440 - val_acc: 0.8475\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fa99c562e3384a118195d8bd4d269025"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 93/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.5002 - acc: 0.8281Epoch 00092: val_loss improved from 0.44241 to 0.44151, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 26s - loss: 0.5001 - acc: 0.8281 - val_loss: 0.4415 - val_acc: 0.8508\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1caf5f0eb9f34c9b9dee96dc3bccdac2"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 94/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.4992 - acc: 0.8282Epoch 00093: val_loss improved from 0.44151 to 0.43424, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 25s - loss: 0.4994 - acc: 0.8282 - val_loss: 0.4342 - val_acc: 0.8525\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c6ade66666324a89abc401566b16a5b9"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 95/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.5017 - acc: 0.8271Epoch 00094: val_loss did not improve\n",
      "781/781 [==============================] - 25s - loss: 0.5016 - acc: 0.8272 - val_loss: 0.4450 - val_acc: 0.8479\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e27fd3a613b14cdd854ca8a80c189fe1"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 96/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.4971 - acc: 0.8284Epoch 00095: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.4970 - acc: 0.8284 - val_loss: 0.4349 - val_acc: 0.8533\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8a60ef49d31f41bc8cfdd27e6d3bca07"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 97/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.4942 - acc: 0.8301Epoch 00096: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.4942 - acc: 0.8301 - val_loss: 0.4476 - val_acc: 0.8501\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "192c1180384846b29d617babcbd66a1e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 98/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.4884 - acc: 0.8317Epoch 00097: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.4882 - acc: 0.8318 - val_loss: 0.4421 - val_acc: 0.8498\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "531f5d82eb6e460eb6669750a41deb8d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 99/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.4918 - acc: 0.8305Epoch 00098: val_loss improved from 0.43424 to 0.43413, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 26s - loss: 0.4918 - acc: 0.8305 - val_loss: 0.4341 - val_acc: 0.8525\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4b7ed1689b3e4abcaa99a2ff04b3ba75"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 100/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.4913 - acc: 0.8305Epoch 00099: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.4914 - acc: 0.8306 - val_loss: 0.4364 - val_acc: 0.8513\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aaece3f191fd4bc0b4e6e8bba82a163f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 101/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.4830 - acc: 0.8320- ETA: 1s -Epoch 00100: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.4830 - acc: 0.8320 - val_loss: 0.4419 - val_acc: 0.8514\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "76bb3908edf94a2da689c8ab094186a2"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 102/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.4892 - acc: 0.8311- ETA: 0s - loss: 0.4888 - acc:Epoch 00101: val_loss improved from 0.43413 to 0.42264, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 25s - loss: 0.4891 - acc: 0.8311 - val_loss: 0.4226 - val_acc: 0.8576\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2e3206d98e604235b6c0c724dad1a29f"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 103/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.4852 - acc: 0.8322Epoch 00102: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.4851 - acc: 0.8323 - val_loss: 0.4304 - val_acc: 0.8545\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5a8e44d05f3545169c7ac3c2a5badfc2"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 104/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.4808 - acc: 0.8343- ETA: 1s - loss: Epoch 00103: val_loss did not improve\n",
      "781/781 [==============================] - 25s - loss: 0.4809 - acc: 0.8342 - val_loss: 0.4276 - val_acc: 0.8549\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "89e37d8da04a4846a0755c55f5ecd253"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 105/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.4826 - acc: 0.8328Epoch 00104: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.4828 - acc: 0.8327 - val_loss: 0.4379 - val_acc: 0.8519\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ffc32e288e0d4b60b69d0df1cfcfbd65"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 106/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.4794 - acc: 0.8352Epoch 00105: val_loss did not improve\n",
      "781/781 [==============================] - 25s - loss: 0.4795 - acc: 0.8352 - val_loss: 0.4358 - val_acc: 0.8539\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4aa1ea1062384ff1ade185208c2b0b2e"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 107/200\n",
      "778/781 [============================>.] - ETA: 0s - loss: 0.4806 - acc: 0.8342Epoch 00106: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.4807 - acc: 0.8341 - val_loss: 0.4267 - val_acc: 0.8559\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "54b7b7ff3c2d4a2a9b73e0f13350a23d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 108/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.4792 - acc: 0.8349Epoch 00107: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.4791 - acc: 0.8348 - val_loss: 0.4400 - val_acc: 0.8506\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cb4fdbf0c2384772bdb8bdca94375be0"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 109/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.4781 - acc: 0.8359- ETA: 0s - loss: 0.4777 - acc:Epoch 00108: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.4784 - acc: 0.8358 - val_loss: 0.4347 - val_acc: 0.8567\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "39d4ae2727484bb38364676a3f5db238"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 110/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.4762 - acc: 0.8359Epoch 00109: val_loss did not improve\n",
      "781/781 [==============================] - 25s - loss: 0.4764 - acc: 0.8358 - val_loss: 0.4244 - val_acc: 0.8576\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8d18892b496a43b0b6cb937de2d1a50d"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 111/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.4752 - acc: 0.8363Epoch 00110: val_loss improved from 0.42264 to 0.42130, saving model to model_checkpoint_cifar110_vgg8_pn_0.2.h5\n",
      "781/781 [==============================] - 26s - loss: 0.4751 - acc: 0.8364 - val_loss: 0.4213 - val_acc: 0.8568\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "aa29b28e85fa4b428e9f3101f2970f98"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 112/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.4748 - acc: 0.8370Epoch 00111: val_loss did not improve\n",
      "781/781 [==============================] - 25s - loss: 0.4751 - acc: 0.8370 - val_loss: 0.4315 - val_acc: 0.8552\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "de5fce0c6e0148b683551279c22cc73a"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 113/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.4691 - acc: 0.8372Epoch 00112: val_loss did not improve\n",
      "781/781 [==============================] - 25s - loss: 0.4690 - acc: 0.8372 - val_loss: 0.4265 - val_acc: 0.8575\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "adbc655ed3d54762a65490b4127e1102"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 114/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.4688 - acc: 0.8371Epoch 00113: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.4689 - acc: 0.8371 - val_loss: 0.4288 - val_acc: 0.8548\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a635c33e1d624ed2a7e20257eebe4f9c"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 115/200\n",
      "780/781 [============================>.] - ETA: 0s - loss: 0.4700 - acc: 0.8381Epoch 00114: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.4700 - acc: 0.8381 - val_loss: 0.4299 - val_acc: 0.8539\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d2e7ae20955b41c3ad8d6eb9774c7edf"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 116/200\n",
      "778/781 [============================>.] - ETA: 0s - loss: 0.4662 - acc: 0.8377Epoch 00115: val_loss did not improve\n",
      "781/781 [==============================] - 28s - loss: 0.4666 - acc: 0.8376 - val_loss: 0.4342 - val_acc: 0.8544\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bad1e49a4c8f409e856d54b11a56b894"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 117/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.4661 - acc: 0.8388Epoch 00116: val_loss did not improve\n",
      "781/781 [==============================] - 26s - loss: 0.4662 - acc: 0.8388 - val_loss: 0.4253 - val_acc: 0.8587\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4fff84cafa7147ec90e615f9ba45fd18"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 118/200\n",
      "779/781 [============================>.] - ETA: 0s - loss: 0.4664 - acc: 0.8382Epoch 00117: val_loss did not improve\n",
      "781/781 [==============================] - 27s - loss: 0.4664 - acc: 0.8382 - val_loss: 0.4425 - val_acc: 0.8531\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b0f00cbb6dcf4952956528fd1294a6c8"
      }
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 119/200\n",
      "438/781 [===============>..............] - ETA: 11s - loss: 0.4650 - acc: 0.8390"
     ]
    }
   ],
   "source": [
    "print('Training {} samples'.format(x_train_enc.shape[0]))\n",
    "\n",
    "if not data_augmentation:\n",
    "    print('No data augmentation applied.')\n",
    "    model.fit(x_train_enc, y_train_enc,\n",
    "              batch_size=batch_size,\n",
    "              epochs=epochs,\n",
    "              validation_data=(x_test, y_test),\n",
    "              shuffle=True,\n",
    "              callbacks=[csv_logger, checkpointer, early_stopper]) \n",
    "else:\n",
    "    print('Using real-time data augmentation.')\n",
    "    # This will do preprocessing and realtime data augmentation:\n",
    "    datagen = ImageDataGenerator(\n",
    "        featurewise_center=False,  # set input mean to 0 over the dataset\n",
    "        samplewise_center=False,  # set each sample mean to 0\n",
    "        featurewise_std_normalization=False,  # divide inputs by std\n",
    "        samplewise_std_normalization=False,  # divide each input by its std\n",
    "        zca_whitening=False,  # apply ZCA whitening\n",
    "        # randomly rotate images in the range (degrees, 0 to 180)\n",
    "        rotation_range=0,\n",
    "        # randomly shift images horizontally (fraction of total width)\n",
    "        width_shift_range=0.1,\n",
    "        # randomly shift images vertically (fraction of total height)\n",
    "        height_shift_range=0.1,\n",
    "        horizontal_flip=True,  # randomly flip images\n",
    "        vertical_flip=False)  # randomly flip images\n",
    "\n",
    "    # Compute quantities required for feature-wise normalization\n",
    "    # (std, mean, and principal components if ZCA whitening is applied).\n",
    "    datagen.fit(x_train)\n",
    "\n",
    "    # Fit the model on the batches generated by datagen.flow().\n",
    "    model.fit_generator(datagen.flow(x_train_enc, y_train_enc,\n",
    "                                     batch_size=batch_size),\n",
    "                        steps_per_epoch=x_train.shape[0] // batch_size,\n",
    "                        epochs=epochs,\n",
    "                        validation_data=(x_test, y_test),\n",
    "                        class_weight=class_weight,\n",
    "                        callbacks=[csv_logger, checkpointer, early_stopper,\n",
    "                                   lr_reducer, TQDMNotebookCallback()])\n",
    "    model.save('{}_pn_{}.h5'.format(title, pct_missing))\n",
    "    \n",
    "    # Predict\n",
    "    y_train_pred_enc = model.predict(x_train)\n",
    "    y_test_pred_enc = model.predict(x_test)\n",
    "    \n",
    "    lst = []\n",
    "    y_train_pred = np.argmax(y_train_pred_enc, axis=-1)\n",
    "    y_train_true = np.argmax(y_train, axis=-1)\n",
    "    y_train_label = np.argmax(y_train_pu[pct_missing], axis=-1)\n",
    "    for y_pred, y_true, y_label in zip(y_train_pred, y_train_true, y_train_label):\n",
    "        lst.append(dict(y_pred=y_pred, y_true=y_true, y_label=y_label, split='train'))\n",
    "\n",
    "    y_test_pred = np.argmax(y_test_pred_enc, axis=-1)\n",
    "    y_test_true = np.argmax(y_test, axis=-1)\n",
    "    for y_pred, y_true in zip(y_test_pred, y_test_true):\n",
    "        lst.append(dict(y_pred=y_pred, y_true=y_true, y_label=y_true, split='test'))\n",
    "    \n",
    "    df = pd.DataFrame(lst)\n",
    "    df.to_csv('prediction_{}_pn_{}.csv'.format(title, pct_missing), index=False, mode='a')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
